#!/usr/bin/python -i

import sys, os, re
import time
import urllib
import StringIO
from lxml import etree
from dateutil import parser as dateparser

import cPickle
import bsddb as bdb

parser = etree.HTMLParser()
topuri = "http://webarchives.loc.gov/collections/"

try:
    fh = urllib.urlopen(topuri)
except:
    print "Couldn't retrieve IA data"

data = fh.read()
fh.close()

try:
    dom = etree.parse(StringIO.StringIO(data), parser)
except:
    print "Not parsable"

linkre = re.compile('lcwa[0-9]+')
links = dom.xpath('//a/@href')
collections = []
for l in links:
    if linkre.match(l):
        collections.append(l[:-1])

linkCollHash = {}

for c in collections:
    print "Processing Collection: %s" % c
    uri = topuri + c +"/"

    try:
        fh = urllib.urlopen(uri)
    except:
        print "Couldn't retrieve IA data"

    data = fh.read()
    fh.close()

    try:
        dom = etree.parse(StringIO.StringIO(data), parser)
    except:
        print "Not parsable"

    links = dom.xpath('//a/@href')
    for l in links:
        base = '../%s/*/' % c
	base2 = './*/'
        if l.startswith(base) or l.startswith(base2):
            if l.startswith(base):
               href = l[len(base):]
            else:
               href=l[len(base2):]
            if href:
                if not href.startswith('http://'):
                    href = 'http://' + href
                try:
                    linkCollHash[href].append(str(c))
                except:
                    linkCollHash[href] = [str(c)]



cxn = bdb.db.DB()
dbPath = "locHash.bdb"
cxn.open(dbPath, dbtype=bdb.db.DB_BTREE, flags = bdb.db.DB_CREATE, mode=0660)
for (k,v) in linkCollHash.iteritems():
    try:
        cxn.put(k, cPickle.dumps(v))
    except:
        cxn.put(k.encode('utf8'), cPickle.dumps(v))
cxn.close()


